#!/bin/bash
flume_version="1.9.0"

flume_url="https://mirrors.huaweicloud.com/apache/flume/${flume_version}/apache-flume-${flume_version}-bin.tar.gz"

wget -c ${flume_url}

tar zxvf apache-flume-${flume_version}-bin.tar.gz || exit 1

mv apache-flume-${flume_version}-bin /opt/ || exit 1

cd /opt

ln -s apache-flume-${flume_version}-bin flume || exit 1

# must install jdk first and set jdk environment for flume
cat >/opt/flume/flume-env.sh <<EOF
export JAVA_HOME=/opt/jdk
export JAVA_OPTS="-Xms100m -Xmx2048m -Dcom.sun.management.jmxremote"
EOF
# create template configure , modify it According to your environment
cat >/opt/flume/conf/tail_to_kafka.conf <<EOF
a1.sources = r1
a1.sinks = k1
a1.channels = c1
a1.sources.r1.type = exec
a1.sources.r1.channels = c1
a1.sources.r1.command = tail -f /data/nginx/log/access.log
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = nginxlog
a1.sinks.k1.brokerList = kafka.example.com:9092
a1.sinks.k1.requiredAcks = 1
a1.sinks.k1.batchSize = 20
a1.sinks.k1.channel = c1
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
EOF

#create startup script
cat >/opt/flume/start_tail_to_kafka.sh <<EOF
#!/bin/bash
cd /opt/flume
bin/flume-ng agent -c conf/ -f conf/tail_to_kafka.conf -n a1 -Dflume.root.logger=INFO,console   -Dflume.monitoring.type=http -Dflume.monitoring.port=34545 >> /tmp/flume_tail_to_kafka.log 2>&1 &
EOF

chmod +x /opt/flume/start_tail_to_kafka.sh
